{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "stuck-integrity",
   "metadata": {},
   "source": [
    "# K8S Resource Test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alone-milan",
   "metadata": {},
   "outputs": [],
   "source": [
    "SCHEDULER_IP=!kubectl get svc seldon-scheduler -n seldon-mesh -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n",
    "SCHEDULER_IP=SCHEDULER_IP[0]\n",
    "import os\n",
    "os.environ['SCHEDULER_IP'] = SCHEDULER_IP\n",
    "SCHEDULER_IP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alternative-dryer",
   "metadata": {},
   "outputs": [],
   "source": [
    "MESH_IP=!kubectl get svc seldon-mesh -n seldon-mesh -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n",
    "MESH_IP=MESH_IP[0]\n",
    "import os\n",
    "os.environ['MESH_IP'] = MESH_IP\n",
    "MESH_IP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "proprietary-mason",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from collections import defaultdict\n",
    "def testJson(model, iterations):\n",
    "    cnts = defaultdict(int)\n",
    "    %env model=$model\n",
    "    for i in range(iterations):\n",
    "        res = !curl -s http://${MESH_IP}/v2/models/${model}/infer -H \"Content-Type: application/json\" -H \"seldon-model: ${model}\"\\\n",
    "            -d '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}'\n",
    "        res = json.loads(res[0])\n",
    "        cnts[res[\"model_name\"]] += 1\n",
    "    for name,cnt in cnts.items():\n",
    "        print(name,cnt)\n",
    "        \n",
    "def testGrpc(model, iterations):\n",
    "    cnts = defaultdict(int)\n",
    "    %env model=$model\n",
    "    for i in range(iterations):\n",
    "        res = !grpcurl -d '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' \\\n",
    "              -plaintext \\\n",
    "              -import-path ../../apis \\\n",
    "              -proto ../../apis/mlops/v2_dataplane/v2_dataplane.proto \\\n",
    "              -rpc-header \"seldon-model: ${model}\" \\\n",
    "              ${MESH_IP}:80 inference.GRPCInferenceService/ModelInfer\n",
    "        res = json.loads(''.join(res))\n",
    "        cnts[res[\"modelName\"]] += 1       \n",
    "    for name,cnt in cnts.items():\n",
    "        print(name,cnt)\n",
    "        \n",
    "def testExperimentJson(experiment, iterations):\n",
    "    cnts = defaultdict(int)\n",
    "    %env model=$experiment\n",
    "    for i in range(iterations):\n",
    "        res = !curl -s http://${MESH_IP}:80/v2/models/exp1/infer -H \"Content-Type: application/json\" -H \"seldon-model: ${model}.experiment\"\\\n",
    "            -d '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}'\n",
    "        res = json.loads(res[0])\n",
    "        cnts[res[\"model_name\"]] += 1\n",
    "    for name,cnt in cnts.items():\n",
    "        print(name,cnt)\n",
    "        \n",
    "def testExperimentGrpc(experiment, iterations):\n",
    "    cnts = defaultdict(int)\n",
    "    %env model=$experiment\n",
    "    for i in range(iterations):\n",
    "        res = !grpcurl -d '{\"model_name\":\"exp1\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' \\\n",
    "              -plaintext \\\n",
    "              -import-path ../../apis \\\n",
    "              -proto ../../apis/mlops/v2_dataplane/v2_dataplane.proto \\\n",
    "              -rpc-header \"seldon-model:${model}.experiment\" \\\n",
    "              ${MESH_IP}:80 inference.GRPCInferenceService/ModelInfer\n",
    "        res = json.loads(''.join(res))\n",
    "        cnts[res[\"modelName\"]] += 1       \n",
    "    for name,cnt in cnts.items():\n",
    "        print(name,cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "romantic-fundamentals",
   "metadata": {},
   "source": [
    "## AB Test with iris default model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "physical-visiting",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "french-convergence",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "blank-composite",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amino-alpha",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "liable-astronomy",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unsigned-adolescent",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/ab-default-model.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "breathing-emperor",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/ab-default-model.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "medieval-shopper",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s experiment --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "structural-amount",
   "metadata": {},
   "outputs": [],
   "source": [
    "testJson(\"iris\",50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "willing-doubt",
   "metadata": {},
   "outputs": [],
   "source": [
    "testGrpc(\"iris\",50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "saving-laptop",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/ab-default-model.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "generic-warehouse",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "expired-facial",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "immune-retro",
   "metadata": {},
   "source": [
    "## Experiment over two iris models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pharmaceutical-weapon",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "religious-corruption",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "focal-academy",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "corrected-sailing",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "false-reason",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "false-korean",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat ../../operator/samples/experiments/ab.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "attractive-biography",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl create -f ../../operator/samples/experiments/ab.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "subsequent-enterprise",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s experiment --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "authentic-orchestra",
   "metadata": {},
   "outputs": [],
   "source": [
    "testJson(\"iris\",50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "first-relations",
   "metadata": {},
   "outputs": [],
   "source": [
    "testExperimentJson(\"experiment-iris\", 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "russian-restriction",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/ab.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "liberal-furniture",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "guilty-paper",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl delete -f ../../operator/samples/experiments/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "exclusive-shower",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
